{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import clone\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "from xgboost import XGBClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from lightgbm import early_stopping  \n",
    "from catboost import CatBoostClassifier, Pool\n",
    "\n",
    "\n",
    "def cross_validate_score(\n",
    "    model, \n",
    "    data: pd.DataFrame, \n",
    "    cv=None,\n",
    "    test_data: pd.DataFrame = None, \n",
    "    label: str = 'Response'\n",
    ") -> tuple[list, np.ndarray, np.ndarray]:\n",
    "    \"\"\"\n",
    "    执行带早停的交叉验证并生成预测结果\n",
    "    \n",
    "    参数:\n",
    "        model: 支持早停的分类器（XGBoost/LightGBM/CatBoost）\n",
    "        data: 包含特征和标签的训练数据\n",
    "        cv: 交叉验证策略，默认分层5折\n",
    "        test_data: 外部测试数据集\n",
    "        label: 目标列名\n",
    "        \n",
    "    返回:\n",
    "        val_scores: 各折验证集AUC得分列表\n",
    "        val_predictions: 验证集样本的OOF预测概率\n",
    "        test_predictions: 测试集预测概率（各折平均）\n",
    "    \"\"\"\n",
    "    # 初始化交叉验证策略\n",
    "    if cv is None:\n",
    "        cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "        \n",
    "    if test_data is None:\n",
    "        raise ValueError(\"必须提供测试数据集 test_data\")\n",
    "\n",
    "    # 分离特征和标签\n",
    "    X = data.copy()  # 避免修改原始数据\n",
    "    y = X.pop(label)\n",
    "\n",
    "    # 初始化预测数组\n",
    "    val_predictions = np.zeros(len(X))\n",
    "    test_predictions = np.zeros(len(test_data))\n",
    "    train_scores, val_scores = [], []\n",
    "\n",
    "    for fold, (train_idx, val_idx) in enumerate(cv.split(X, y)):\n",
    "        # 划分训练集/验证集\n",
    "        X_train = X.iloc[train_idx].reset_index(drop=True)\n",
    "        y_train = y.iloc[train_idx].reset_index(drop=True)\n",
    "        X_val = X.iloc[val_idx].reset_index(drop=True)\n",
    "        y_val = y.iloc[val_idx].reset_index(drop=True)\n",
    "\n",
    "        # 克隆模型保证每次迭代独立\n",
    "        model_clone = clone(model)\n",
    "\n",
    "        # 根据模型类型进行差异化处理\n",
    "        if isinstance(model_clone, XGBClassifier):\n",
    "            \"\"\"XGBoost处理逻辑\"\"\"\n",
    "            eval_set = [(X_val, y_val)]\n",
    "            model_clone.fit(\n",
    "                X_train, y_train,\n",
    "                eval_set=eval_set,\n",
    "                verbose=False\n",
    "            )\n",
    "            best_iter = model_clone.best_iteration\n",
    "            pred_args = {'iteration_range': (0, best_iter)}\n",
    "\n",
    "        elif isinstance(model_clone, LGBMClassifier):\n",
    "            \"\"\"LightGBM处理逻辑\"\"\"\n",
    "            eval_set = [(X_val, y_val)]\n",
    "            model_clone.fit(\n",
    "                X_train, y_train,\n",
    "                eval_set=eval_set,\n",
    "                eval_metric='auc',\n",
    "                callbacks=[early_stopping(50)]\n",
    "            )\n",
    "            best_iter = model_clone.best_iteration_\n",
    "            pred_args = {'num_iteration': best_iter}\n",
    "\n",
    "        elif isinstance(model_clone, CatBoostClassifier):\n",
    "            \"\"\"CatBoost处理逻辑\"\"\"\n",
    "            # 使用Pool提高内存效率\n",
    "            train_pool = Pool(X_train, y_train)\n",
    "            val_pool = Pool(X_val, y_val)\n",
    "            test_pool = Pool(test_data)\n",
    "            \n",
    "            model_clone.fit(\n",
    "                train_pool,\n",
    "                eval_set=val_pool,\n",
    "                early_stopping_rounds=50,\n",
    "                verbose=False\n",
    "            )\n",
    "            best_iter = model_clone.get_best_iteration()\n",
    "            pred_args = {}  # CatBoost自动使用最佳迭代\n",
    "            \n",
    "            # 及时释放Pool内存\n",
    "            del train_pool, val_pool, test_pool\n",
    "\n",
    "        else:\n",
    "            raise ValueError(\"仅支持XGBoost、LightGBM、CatBoost分类器\")\n",
    "\n",
    "        # 生成预测概率\n",
    "        train_preds = model_clone.predict_proba(X_train, **pred_args)[:, 1]\n",
    "        val_preds = model_clone.predict_proba(X_val, **pred_args)[:, 1]\n",
    "        test_preds = model_clone.predict_proba(test_data, **pred_args)[:, 1]\n",
    "\n",
    "        # 记录分数和预测结果\n",
    "        val_predictions[val_idx] = val_preds\n",
    "        train_scores.append(roc_auc_score(y_train, train_preds))\n",
    "        val_scores.append(roc_auc_score(y_val, val_preds))\n",
    "        test_predictions += test_preds / cv.get_n_splits()\n",
    "\n",
    "        # 释放内存\n",
    "        del model_clone, X_train, y_train, X_val, y_val\n",
    "        gc.collect()\n",
    "\n",
    "        # 打印本折结果\n",
    "        print(f'Fold {fold}: Val AUC = {val_scores[-1]:.5f}')\n",
    "\n",
    "    # 打印汇总结果\n",
    "    print(f'\\n验证集平均 AUC: {np.mean(val_scores):.5f} ± {np.std(val_scores):.5f}')\n",
    "    print(f'训练集平均 AUC: {np.mean(train_scores):.5f} ± {np.std(train_scores):.5f}')\n",
    "    \n",
    "    return val_scores, val_predictions, test_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_classification\n",
    "\n",
    "# 生成二分类数据集\n",
    "X, y = make_classification(\n",
    "    n_samples=1000, \n",
    "    n_features=20, \n",
    "    n_informative=15, \n",
    "    n_classes=2, \n",
    "    random_state=42,\n",
    "    \n",
    ")\n",
    "data = pd.DataFrame(X, columns=[f'feature_{i}' for i in range(20)])\n",
    "data['Response'] = y\n",
    "\n",
    "# 划分训练集和测试集\n",
    "test_data = data.sample(frac=0.2, random_state=42)\n",
    "train_data = data.drop(test_data.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_params = {\n",
    "    # 核心参数\n",
    "    'objective': 'binary:logistic',  # 目标函数：二分类逻辑回归\n",
    "    'eval_metric': 'auc',            # 评估指标：AUC（适用于分类任务）\n",
    "    'learning_rate': 0.075,          # 学习率：控制每棵树的贡献，越小越慢但精度越高\n",
    "    'n_estimators': 5000,            # 树的数量上限：需配合早停使用\n",
    "    \n",
    "    # 树结构控制\n",
    "    'max_depth': 9,                  # 树的最大深度：控制模型复杂度，越大越容易过拟合\n",
    "    'max_leaves': 512,               # 最大叶子节点数：用于近似树方法（hist）\n",
    "    'min_child_weight': 10,          # 叶节点最小样本权重和：防止过拟合\n",
    "    'gamma': 0.1,                    # 分裂所需最小增益：值越大模型越保守\n",
    "    \n",
    "    # 正则化参数\n",
    "    'reg_lambda': 0.5,               # L2正则化系数：控制权重平方和惩罚\n",
    "    'reg_alpha': 0.1,                # L1正则化系数：控制权重绝对值惩罚\n",
    "    'subsample': 0.8,                # 样本采样比例：增强多样性，防止过拟合\n",
    "    'colsample_bytree': 0.8,         # 特征采样比例：增强多样性，防止过拟合\n",
    "    \n",
    "    # 训练策略\n",
    "    'scale_pos_weight': 1.0,         # 正样本权重：用于处理类别不平衡问题\n",
    "    \n",
    "    # 系统优化\n",
    "    # 'tree_method': 'gpu_hist',       # 树方法：使用GPU加速的直方图算法\n",
    "    'random_state': 42,              # 随机种子：保证结果可复现\n",
    "}\n",
    "\n",
    "lgbm_params = {\n",
    "    # 核心参数\n",
    "    'objective': 'binary',           # 目标函数：二分类任务\n",
    "    'metric': 'auc',                 # 评估指标：AUC\n",
    "    'learning_rate': 0.075,          # 学习率：控制每棵树的贡献\n",
    "    'n_estimators': 5000,            # 树的数量上限：需配合早停使用\n",
    "    \n",
    "    # 树结构控制\n",
    "    'max_depth': 9,                  # 树的最大深度：控制模型复杂度\n",
    "    'num_leaves': 512,               # 最大叶子节点数：需满足 2^max_depth ≥ num_leaves\n",
    "    'min_child_samples': 20,         # 叶节点最小样本数：防止过拟合\n",
    "    'min_split_gain': 0.1,           # 分裂所需最小增益：值越大模型越保守\n",
    "    \n",
    "    # 正则化参数\n",
    "    'lambda_l2': 0.5,                # L2正则化系数：控制权重平方和惩罚\n",
    "    'lambda_l1': 0.1,                # L1正则化系数：控制权重绝对值惩罚\n",
    "    'bagging_fraction': 0.8,         # 样本采样比例：增强多样性\n",
    "    'feature_fraction': 0.8,         # 特征采样比例：增强多样性\n",
    "    \n",
    "    # 训练策略\n",
    "    'early_stopping_rounds': 50,     # 早停轮数：验证集指标无提升时停止训练\n",
    "    'class_weight': 'balanced',      # 类别权重：处理类别不平衡问题\n",
    "    \n",
    "    # 系统优化\n",
    "    # 'device_type': 'gpu',            # 设备类型：使用GPU加速\n",
    "    'random_state': 42,              # 随机种子：保证结果可复现\n",
    "    'verbose': -1                    # 日志输出级别：-1=静默，0=警告，1=信息\n",
    "}\n",
    "\n",
    "catb_params = {\n",
    "    # 核心参数\n",
    "    'loss_function': 'Logloss',      # 损失函数：对数损失（二分类）\n",
    "    'eval_metric': 'AUC',            # 评估指标：AUC\n",
    "    'learning_rate': 0.075,          # 学习率：控制每棵树的贡献\n",
    "    'iterations': 5000,              # 最大迭代次数：需配合早停使用\n",
    "    \n",
    "    # 树结构控制\n",
    "    'depth': 9,                      # 树的最大深度：控制模型复杂度\n",
    "    'l2_leaf_reg': 0.5,              # L2正则化系数：控制权重平方和惩罚\n",
    "    'min_data_in_leaf': 20,          # 叶节点最小样本数：防止过拟合\n",
    "    'grow_policy': 'SymmetricTree',  # 树生长策略：对称树（默认）\n",
    "    \n",
    "    # 采样与正则化\n",
    "    'bootstrap_type': 'Bernoulli',   # 自助法类型：伯努利采样\n",
    "    'subsample': 0.8,                # 样本采样比例：增强多样性\n",
    "    'rsm': 0.8,                      # 特征采样比例：增强多样性\n",
    "    'random_strength': 1.0,          # 随机强度：控制分裂时的随机性\n",
    "    \n",
    "    # 训练策略\n",
    "    'early_stopping_rounds': 50,     # 早停轮数：验证集指标无提升时停止训练\n",
    "    'class_weights': [0.5, 2],       # 类别权重：处理类别不平衡问题\n",
    "    \n",
    "    # 系统优化\n",
    "    # 'task_type': 'GPU',              # 任务类型：使用GPU加速\n",
    "    'random_seed': 42,               # 随机种子：保证结果可复现\n",
    "    'verbose': False                 # 日志输出：False=静默，True=输出日志\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# XGBoost（需指定 eval_metric）\n",
    "xgb_model = XGBClassifier(\n",
    "    early_stopping_rounds=50,\n",
    "    **xgb_params\n",
    ")\n",
    "\n",
    "# LightGBM\n",
    "lgbm_model =  LGBMClassifier(\n",
    "    **lgbm_params\n",
    ")\n",
    "\n",
    "# CatBoost\n",
    "catboost_model = CatBoostClassifier(\n",
    "    **catb_params\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== XGBoost ===\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n",
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "\n",
      "=== LightGBM ===\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "Early stopping, best iteration is:\n",
      "[103]\tvalid_0's auc: 0.967141\n",
      "Fold 0: Val AUC = 0.96714\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "Early stopping, best iteration is:\n",
      "[113]\tvalid_0's auc: 0.965134\n",
      "Fold 1: Val AUC = 0.96513\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "Fold 2: Val AUC = 0.95091\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "Early stopping, best iteration is:\n",
      "[109]\tvalid_0's auc: 0.973421\n",
      "Fold 3: Val AUC = 0.97342\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "Fold 4: Val AUC = 0.96248\n",
      "\n",
      "验证集平均 AUC: 0.96382 ± 0.00740\n",
      "训练集平均 AUC: 1.00000 ± 0.00000\n",
      "\n",
      "=== CatBoost ===\n",
      "Fold 0: Val AUC = 0.98216\n",
      "Fold 1: Val AUC = 0.97545\n",
      "Fold 2: Val AUC = 0.96967\n",
      "Fold 3: Val AUC = 0.97921\n",
      "Fold 4: Val AUC = 0.97373\n",
      "\n",
      "验证集平均 AUC: 0.97604 ± 0.00433\n",
      "训练集平均 AUC: 1.00000 ± 0.00000\n"
     ]
    }
   ],
   "source": [
    "# 运行交叉验证\n",
    "print(\"=== XGBoost ===\")\n",
    "xgb_val_scores, xgb_val_preds, xgb_test_preds = cross_validate_score(\n",
    "    xgb_model, \n",
    "    train_data, \n",
    "    test_data=test_data.drop('Response', axis=1), \n",
    "    label='Response'\n",
    ")\n",
    "\n",
    "print(\"\\n=== LightGBM ===\")\n",
    "lgbm_val_scores, lgbm_val_preds, lgbm_test_preds = cross_validate_score(\n",
    "    lgbm_model, \n",
    "    train_data, \n",
    "    test_data=test_data.drop('Response', axis=1), \n",
    "    label='Response'\n",
    ")\n",
    "\n",
    "\n",
    "print(\"\\n=== CatBoost ===\")\n",
    "# X_train_pool = Pool(train_data, label='Response', cat_features=train_data.columns.values[0:3])\n",
    "catboost_val_scores, catboost_val_preds, catboost_test_preds = cross_validate_score(\n",
    "    catboost_model, \n",
    "    train_data, \n",
    "    test_data=test_data.drop('Response', axis=1), \n",
    "    label='Response'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_summary, oof_predictions, test_predictions = pd.DataFrame(), pd.DataFrame(), pd.DataFrame()\n",
    "cv_summary['xgb'], oof_predictions['xgb'], test_predictions['xgb'] = xgb_val_scores, xgb_val_preds, xgb_test_preds\n",
    "cv_summary['catb'], oof_predictions['catb'], test_predictions['catb'] = catboost_val_scores, catboost_val_preds, catboost_test_preds\n",
    "cv_summary['lgbm'], oof_predictions['lgbm'], test_predictions['lgbm'] = lgbm_val_scores, lgbm_val_preds, lgbm_test_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9500873746686265"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(xgb_val_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:03:17,470] A new study created in memory with name: no-name-d734e5af-7f8f-4d9b-851e-39a6643b7167\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:03:20,100] Trial 0 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.08775771542150176, 'alpha': 0.018533664330825855, 'colsample_bytree': 0.5, 'subsample': 0.5, 'learning_rate': 0.016, 'n_estimators': 884, 'max_depth': 5, 'random_state': 2020, 'min_child_weight': 238}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:03:22,672] Trial 1 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.5165457480628188, 'alpha': 0.27224397588815524, 'colsample_bytree': 1.0, 'subsample': 0.4, 'learning_rate': 0.012, 'n_estimators': 439, 'max_depth': 11, 'random_state': 2020, 'min_child_weight': 47}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:03:25,218] Trial 2 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.002357488458246465, 'alpha': 1.3085406066803327, 'colsample_bytree': 0.7, 'subsample': 0.5, 'learning_rate': 0.02, 'n_estimators': 312, 'max_depth': 17, 'random_state': 2020, 'min_child_weight': 249}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:03:27,767] Trial 3 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.0032499526959798622, 'alpha': 0.004739826702618277, 'colsample_bytree': 0.9, 'subsample': 0.4, 'learning_rate': 0.018, 'n_estimators': 496, 'max_depth': 11, 'random_state': 2020, 'min_child_weight': 114}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:03:30,503] Trial 4 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.013515941648590418, 'alpha': 0.33971536853211, 'colsample_bytree': 0.7, 'subsample': 1.0, 'learning_rate': 0.02, 'n_estimators': 946, 'max_depth': 17, 'random_state': 2020, 'min_child_weight': 273}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:03:34,263] Trial 5 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.28376097122563687, 'alpha': 0.0034930400163389955, 'colsample_bytree': 0.4, 'subsample': 0.8, 'learning_rate': 0.018, 'n_estimators': 47, 'max_depth': 17, 'random_state': 2020, 'min_child_weight': 180}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:03:37,967] Trial 6 finished with value: 0.9500873746686265 and parameters: {'lambda': 3.991988453769855, 'alpha': 0.0015173790401320528, 'colsample_bytree': 0.5, 'subsample': 0.6, 'learning_rate': 0.01, 'n_estimators': 635, 'max_depth': 15, 'random_state': 2020, 'min_child_weight': 250}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:03:41,736] Trial 7 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.15441079844386219, 'alpha': 0.0010851700166951798, 'colsample_bytree': 0.7, 'subsample': 0.8, 'learning_rate': 0.014, 'n_estimators': 866, 'max_depth': 5, 'random_state': 2020, 'min_child_weight': 248}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:03:45,613] Trial 8 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.0010976959188844144, 'alpha': 0.45197606810094926, 'colsample_bytree': 0.7, 'subsample': 0.7, 'learning_rate': 0.008, 'n_estimators': 223, 'max_depth': 13, 'random_state': 2020, 'min_child_weight': 275}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:03:49,418] Trial 9 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.2659183930212687, 'alpha': 0.18444602118244843, 'colsample_bytree': 0.4, 'subsample': 0.8, 'learning_rate': 0.012, 'n_estimators': 441, 'max_depth': 5, 'random_state': 2020, 'min_child_weight': 99}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:03:53,169] Trial 10 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.024504268782607343, 'alpha': 0.025987039859910386, 'colsample_bytree': 0.8, 'subsample': 0.5, 'learning_rate': 0.016, 'n_estimators': 742, 'max_depth': 7, 'random_state': 2020, 'min_child_weight': 189}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:03:56,936] Trial 11 finished with value: 0.9500873746686265 and parameters: {'lambda': 2.3264460353624177, 'alpha': 7.5869043771784295, 'colsample_bytree': 1.0, 'subsample': 0.4, 'learning_rate': 0.012, 'n_estimators': 680, 'max_depth': 11, 'random_state': 2020, 'min_child_weight': 4}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:00,675] Trial 12 finished with value: 0.9500873746686265 and parameters: {'lambda': 1.007712626930685, 'alpha': 0.032903100936792204, 'colsample_bytree': 0.5, 'subsample': 0.5, 'learning_rate': 0.016, 'n_estimators': 330, 'max_depth': 9, 'random_state': 2020, 'min_child_weight': 5}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:04,498] Trial 13 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.04948551468033215, 'alpha': 0.03019818556279786, 'colsample_bytree': 0.3, 'subsample': 0.4, 'learning_rate': 0.012, 'n_estimators': 585, 'max_depth': 11, 'random_state': 2020, 'min_child_weight': 47}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:08,359] Trial 14 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.7220881009253979, 'alpha': 0.06131584583104313, 'colsample_bytree': 1.0, 'subsample': 0.6, 'learning_rate': 0.016, 'n_estimators': 833, 'max_depth': 5, 'random_state': 2020, 'min_child_weight': 203}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:12,078] Trial 15 finished with value: 0.9500873746686265 and parameters: {'lambda': 8.728046255408897, 'alpha': 0.009417522507245498, 'colsample_bytree': 0.6, 'subsample': 0.7, 'learning_rate': 0.008, 'n_estimators': 134, 'max_depth': 13, 'random_state': 2020, 'min_child_weight': 128}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:15,831] Trial 16 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.06555648054306588, 'alpha': 1.1970359528729613, 'colsample_bytree': 0.5, 'subsample': 1.0, 'learning_rate': 0.01, 'n_estimators': 999, 'max_depth': 9, 'random_state': 2020, 'min_child_weight': 78}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:19,584] Trial 17 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.6229984404059742, 'alpha': 0.09472510107342394, 'colsample_bytree': 1.0, 'subsample': 0.5, 'learning_rate': 0.014, 'n_estimators': 413, 'max_depth': 15, 'random_state': 2020, 'min_child_weight': 161}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:23,620] Trial 18 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.01118041078588013, 'alpha': 7.481782221169836, 'colsample_bytree': 0.3, 'subsample': 0.4, 'learning_rate': 0.016, 'n_estimators': 548, 'max_depth': 7, 'random_state': 2020, 'min_child_weight': 52}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:28,073] Trial 19 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.11887189507478936, 'alpha': 0.01235300636083887, 'colsample_bytree': 0.6, 'subsample': 0.5, 'learning_rate': 0.012, 'n_estimators': 776, 'max_depth': 5, 'random_state': 2020, 'min_child_weight': 212}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:31,918] Trial 20 finished with value: 0.9500873746686265 and parameters: {'lambda': 1.3761862282310877, 'alpha': 1.1774349451171378, 'colsample_bytree': 0.9, 'subsample': 0.4, 'learning_rate': 0.012, 'n_estimators': 345, 'max_depth': 11, 'random_state': 2020, 'min_child_weight': 144}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:35,843] Trial 21 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.0034660343240906343, 'alpha': 2.0070786588980805, 'colsample_bytree': 0.8, 'subsample': 0.5, 'learning_rate': 0.02, 'n_estimators': 263, 'max_depth': 17, 'random_state': 2020, 'min_child_weight': 234}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:39,606] Trial 22 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.0011992989728933018, 'alpha': 0.459119159692072, 'colsample_bytree': 0.7, 'subsample': 0.5, 'learning_rate': 0.02, 'n_estimators': 215, 'max_depth': 17, 'random_state': 2020, 'min_child_weight': 292}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:44,219] Trial 23 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.0055980700334580845, 'alpha': 2.7446964762769976, 'colsample_bytree': 0.5, 'subsample': 0.5, 'learning_rate': 0.02, 'n_estimators': 408, 'max_depth': 17, 'random_state': 2020, 'min_child_weight': 225}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:47,997] Trial 24 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.033426096316855426, 'alpha': 0.28023968868647636, 'colsample_bytree': 1.0, 'subsample': 0.5, 'learning_rate': 0.02, 'n_estimators': 492, 'max_depth': 5, 'random_state': 2020, 'min_child_weight': 166}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:51,497] Trial 25 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.3001864468993014, 'alpha': 0.7806696264207911, 'colsample_bytree': 1.0, 'subsample': 0.6, 'learning_rate': 0.016, 'n_estimators': 121, 'max_depth': 11, 'random_state': 2020, 'min_child_weight': 299}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:55,341] Trial 26 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.014260196269177221, 'alpha': 0.16425945745660092, 'colsample_bytree': 0.7, 'subsample': 0.7, 'learning_rate': 0.01, 'n_estimators': 302, 'max_depth': 7, 'random_state': 2020, 'min_child_weight': 254}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:04:59,388] Trial 27 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.4071158786026555, 'alpha': 3.003222560523332, 'colsample_bytree': 0.5, 'subsample': 1.0, 'learning_rate': 0.014, 'n_estimators': 620, 'max_depth': 9, 'random_state': 2020, 'min_child_weight': 35}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:05:03,287] Trial 28 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.09556751155121564, 'alpha': 0.08312199687903875, 'colsample_bytree': 0.9, 'subsample': 0.4, 'learning_rate': 0.008, 'n_estimators': 394, 'max_depth': 15, 'random_state': 2020, 'min_child_weight': 79}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:05:07,226] Trial 29 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.0021983303559997163, 'alpha': 0.006304542541332185, 'colsample_bytree': 0.4, 'subsample': 0.4, 'learning_rate': 0.018, 'n_estimators': 485, 'max_depth': 11, 'random_state': 2020, 'min_child_weight': 121}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:05:11,502] Trial 30 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.0068329064956661485, 'alpha': 0.05452578313430258, 'colsample_bytree': 0.8, 'subsample': 0.5, 'learning_rate': 0.018, 'n_estimators': 670, 'max_depth': 13, 'random_state': 2020, 'min_child_weight': 203}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:05:15,403] Trial 31 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.0024971012948921396, 'alpha': 0.0038288583909651406, 'colsample_bytree': 0.9, 'subsample': 0.4, 'learning_rate': 0.018, 'n_estimators': 529, 'max_depth': 11, 'random_state': 2020, 'min_child_weight': 93}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:05:19,396] Trial 32 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.01755544605832902, 'alpha': 0.01613317400075171, 'colsample_bytree': 0.9, 'subsample': 0.4, 'learning_rate': 0.018, 'n_estimators': 462, 'max_depth': 17, 'random_state': 2020, 'min_child_weight': 269}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:05:23,436] Trial 33 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.005054920607380783, 'alpha': 0.0026738000126533402, 'colsample_bytree': 0.9, 'subsample': 1.0, 'learning_rate': 0.02, 'n_estimators': 922, 'max_depth': 11, 'random_state': 2020, 'min_child_weight': 121}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:05:27,573] Trial 34 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.0017680187262388806, 'alpha': 0.0019492760316212744, 'colsample_bytree': 0.7, 'subsample': 0.8, 'learning_rate': 0.018, 'n_estimators': 572, 'max_depth': 17, 'random_state': 2020, 'min_child_weight': 183}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:05:31,400] Trial 35 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.009347684033510908, 'alpha': 0.0056580293997034025, 'colsample_bytree': 0.5, 'subsample': 0.4, 'learning_rate': 0.012, 'n_estimators': 11, 'max_depth': 11, 'random_state': 2020, 'min_child_weight': 141}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:05:35,299] Trial 36 finished with value: 0.9500873746686265 and parameters: {'lambda': 3.342294480581019, 'alpha': 0.16397582194828708, 'colsample_bytree': 0.7, 'subsample': 0.6, 'learning_rate': 0.016, 'n_estimators': 155, 'max_depth': 5, 'random_state': 2020, 'min_child_weight': 268}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:05:39,341] Trial 37 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.21845352295124373, 'alpha': 0.664832615354203, 'colsample_bytree': 0.4, 'subsample': 0.8, 'learning_rate': 0.02, 'n_estimators': 380, 'max_depth': 11, 'random_state': 2020, 'min_child_weight': 29}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:05:43,328] Trial 38 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.0037545164983663333, 'alpha': 0.28318215800750546, 'colsample_bytree': 0.6, 'subsample': 0.7, 'learning_rate': 0.018, 'n_estimators': 733, 'max_depth': 5, 'random_state': 2020, 'min_child_weight': 236}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:05:47,109] Trial 39 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.026559816406107568, 'alpha': 0.019673794153840667, 'colsample_bytree': 0.3, 'subsample': 0.5, 'learning_rate': 0.01, 'n_estimators': 219, 'max_depth': 17, 'random_state': 2020, 'min_child_weight': 67}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:05:50,838] Trial 40 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.0010062168290271372, 'alpha': 0.0075426238698878545, 'colsample_bytree': 1.0, 'subsample': 0.4, 'learning_rate': 0.012, 'n_estimators': 287, 'max_depth': 13, 'random_state': 2020, 'min_child_weight': 101}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:05:54,622] Trial 41 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.046175564659906534, 'alpha': 0.0013170827985040483, 'colsample_bytree': 0.7, 'subsample': 1.0, 'learning_rate': 0.02, 'n_estimators': 994, 'max_depth': 17, 'random_state': 2020, 'min_child_weight': 251}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:05:58,454] Trial 42 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.008663560424832944, 'alpha': 0.3543689264013371, 'colsample_bytree': 0.7, 'subsample': 1.0, 'learning_rate': 0.02, 'n_estimators': 914, 'max_depth': 17, 'random_state': 2020, 'min_child_weight': 285}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:06:02,382] Trial 43 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.16755272916552813, 'alpha': 0.036400217706749234, 'colsample_bytree': 0.7, 'subsample': 1.0, 'learning_rate': 0.016, 'n_estimators': 834, 'max_depth': 15, 'random_state': 2020, 'min_child_weight': 259}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:06:06,123] Trial 44 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.0034637460344422223, 'alpha': 0.18484113555062573, 'colsample_bytree': 0.5, 'subsample': 0.8, 'learning_rate': 0.008, 'n_estimators': 920, 'max_depth': 17, 'random_state': 2020, 'min_child_weight': 222}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:06:09,881] Trial 45 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.07999809858807942, 'alpha': 0.9538219825523211, 'colsample_bytree': 0.7, 'subsample': 0.5, 'learning_rate': 0.014, 'n_estimators': 622, 'max_depth': 5, 'random_state': 2020, 'min_child_weight': 280}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:06:13,563] Trial 46 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.01848909684504453, 'alpha': 1.825040284286064, 'colsample_bytree': 1.0, 'subsample': 0.4, 'learning_rate': 0.02, 'n_estimators': 760, 'max_depth': 9, 'random_state': 2020, 'min_child_weight': 244}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:06:17,471] Trial 47 finished with value: 0.9500873746686265 and parameters: {'lambda': 1.137201324092266, 'alpha': 4.880682919624874, 'colsample_bytree': 0.9, 'subsample': 1.0, 'learning_rate': 0.012, 'n_estimators': 442, 'max_depth': 11, 'random_state': 2020, 'min_child_weight': 193}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:06:21,188] Trial 48 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.5182740599737657, 'alpha': 0.5088905773985735, 'colsample_bytree': 0.8, 'subsample': 0.6, 'learning_rate': 0.016, 'n_estimators': 356, 'max_depth': 7, 'random_state': 2020, 'min_child_weight': 26}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n",
      "Fold 0: Val AUC = 0.95885\n",
      "Fold 1: Val AUC = 0.95310\n",
      "Fold 2: Val AUC = 0.92886\n",
      "Fold 3: Val AUC = 0.96013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-01-26 22:06:24,905] Trial 49 finished with value: 0.9500873746686265 and parameters: {'lambda': 0.0015006151828426378, 'alpha': 0.003975555055147364, 'colsample_bytree': 0.3, 'subsample': 0.7, 'learning_rate': 0.012, 'n_estimators': 871, 'max_depth': 11, 'random_state': 2020, 'min_child_weight': 170}. Best is trial 0 with value: 0.9500873746686265.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4: Val AUC = 0.94950\n",
      "\n",
      "验证集平均 AUC: 0.95009 ± 0.01129\n",
      "训练集平均 AUC: 0.99176 ± 0.00423\n"
     ]
    }
   ],
   "source": [
    "import optuna\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "\n",
    "def objective(trial):\n",
    "    param = {\n",
    "        'tree_method': 'gpu_hist',\n",
    "        'lambda': trial.suggest_float('lambda', 1e-3, 10.0, log=True),\n",
    "        'alpha': trial.suggest_float('alpha', 1e-3, 10.0, log=True),\n",
    "        'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]),\n",
    "        'subsample': trial.suggest_categorical('subsample', [0.4, 0.5, 0.6, 0.7, 0.8, 1.0]),\n",
    "        'learning_rate': trial.suggest_categorical('learning_rate', [0.008, 0.01, 0.012, 0.014, 0.016, 0.018, 0.02]),\n",
    "        'n_estimators': trial.suggest_int('n_estimators', 5, 1000),\n",
    "        'max_depth': trial.suggest_categorical('max_depth', [5, 7, 9, 11, 13, 15, 17]),\n",
    "        'random_state': trial.suggest_categorical('random_state', [2020]),\n",
    "        'min_child_weight': trial.suggest_int('min_child_weight', 1, 300),\n",
    "    }\n",
    "    xgb_val_scores, _, _ = cross_validate_score(\n",
    "        xgb_model, \n",
    "        train_data, \n",
    "        test_data=test_data.drop('Response', axis=1), \n",
    "        label='Response'\n",
    "    )\n",
    "    accuracy = np.mean(xgb_val_scores)\n",
    "    return accuracy\n",
    "\n",
    "\n",
    "study = optuna.create_study(direction='maximize')\n",
    "study.optimize(objective, n_trials=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Hyperparameters: {'lambda': 0.08775771542150176, 'alpha': 0.018533664330825855, 'colsample_bytree': 0.5, 'subsample': 0.5, 'learning_rate': 0.016, 'n_estimators': 884, 'max_depth': 5, 'random_state': 2020, 'min_child_weight': 238}\n",
      "Best Accuracy: 0.950\n"
     ]
    }
   ],
   "source": [
    "best_params = study.best_params\n",
    "best_score = study.best_value\n",
    "print(f\"Best Hyperparameters: {best_params}\")\n",
    "print(f\"Best Accuracy: {best_score:.3f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "kaggle",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
